package com.stockprediction.analysis;


import com.huaban.analysis.jieba.JiebaSegmenter;

import java.util.List;


public class ChineseTokenizer {
    public static String tokenize(String text) {
        if (text == null || text.trim().isEmpty()) {
            return "";
        }
        JiebaSegmenter segmenter = new JiebaSegmenter();
        List<String> words = segmenter.sentenceProcess(text);

        // 过滤掉停用词（可选）
        words.removeIf(word -> word.length() <= 1);

        return String.join(" ", words);
    }
}
